# This file is part of Sonedyan.
#
# Sonedyan is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation;
# either version 3 of the License, or (at your option) any
# later version.
#
# Sonedyan is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2009-2012 Jimmy Dubuisson <jimmy.dubuisson@gmail.com>

#
# usage: python 5a_normalize-filtered-1grams.py
#
# save the normalized time series for each 1gram found in "filtered-1grams.txt" 
#
# NB: creates the file "normalized-filtered-1grams.txt" with the following format:
# <1-gram>, value1, value2, ...

# load the yearly stats
fd1 = open("yearly-stats.txt", "r")

line = fd1.readline().strip()
record = {}

while line:
	year, count = line.split()
	record[year] = int(count) 
	line = fd1.readline().strip()

fd1.close()

# open the non normalized noun stats
fd2 = open("filtered-1grams.txt", "r")

# output file
fd3 = open("normalized-filtered-1grams.txt", "w")

currentNgram = ""

line2 = fd2.readline().strip()

record2 = {}

# read the stats file
while line2:
        ngram, year, matchCount, pageCount, volumeCount = line2.split(',')
	# when we pass to the next 1gram, generate the current 1 gram normalized time series
	if (currentNgram != ngram):
		if (len(record2) > 0):
			timeSeriesValues = currentNgram
			for i in range(1800, 2001):
				if (record2.has_key(str(i))):
					timeSeriesValues += "," + str(record2[str(i)])
				else:
					timeSeriesValues += ",0"
			fd3.write(timeSeriesValues + "\n")
		currentNgram = ngram
	else:	
		if (int(year) >= 1800 and int(year) <= 2000):
        		yearlyCount = record[year]
			record2[year] = float(float(matchCount) / float(record[year]))
	line2 = fd2.readline().strip()

fd2.close()
fd3.close()
